import gzip, MySQLdb
import nltk, json


def read_reviews_from_file(filename):
	with gzip.open(filename,'r') as orfile:
		obj = {}
		line = "start"
		items = []
		while line != '':
			line = orfile.readline()
			buf = [w.strip() for w in line.split(":")]
			try:
				if buf[0] != "":
					obj[buf[0]] = buf[1]
				else:
					yield (obj["review/userId"].strip(), obj["product/productId"].strip(), obj["review/score"].strip())
					obj = {}
			except:
				yield ("unknown", "unknown", "unknown")
				obj = {}

if __name__ == "__main__":
	conn = MySQLdb.connect(host="mysql.stalidis.com", user="ict4growth", passwd="stanford", db="ict4growth")
	print "Connected to database"
	conn.autocommit(False)
	writer = conn.cursor()
	iterator = 0
	Buffer = []
	Users = []
	for review in read_reviews_from_file("BookReviews.txt.gz"):
		iterator += 1
		print "Reading review: ", iterator
		if review[0] != "unknown":
			Buffer.append(review)
			Users.append((review[0],))
		if len(Buffer)%500 == 0:
			writer.executemany("""INSERT IGNORE INTO Books VALUES (%s,%s,%s,0)""", Buffer)
			writer.executemany("""INSERT INTO `NUsers` (`UserID`, `Vector`, `Votes`) VALUES (%s, "NULL", 1) ON DUPLICATE KEY UPDATE `Votes`=`Votes`+1""", Users)
			print "Written ", len(Buffer), " reviews"
			conn.commit()
			Buffer = []
			Users = []
	writer.executemany("""INSERT IGNORE INTO Books VALUES (%s,%s,%s,0)""", Buffer)
	writer.executemany("""INSERT INTO `NUsers` (`UserID`, `Vector`, `Votes`) VALUES (%s, "NULL", 1) ON DUPLICATE KEY UPDATE `Votes`=`Votes`+1""", Users)
	print "Written ", len(Buffer), " reviews"
	conn.commit()
	Buffer = []
	conn.close()


